/*  PageParser.Java
 * 
 *  WebCrawler Program
 *  @author Amandeep Jhajj and Craig Markham
 *  @date 26Apr2012
 */

package Crawler;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Observable;
import java.util.Set;
import java.util.concurrent.BlockingQueue;

public class PageParser extends Observable implements Runnable{

	/** 
	 * A delimiter for punctuation
	 */
	private static final String DELIMITER = "[\\s.\t\",?:!)(\\[\\];\\-\\~]";
	  
	/** 
	 * Hyperlinks Queue 
	 */
    final private BlockingQueue<String> myRetrieveQueue;
    
    /** 
     * Set of pages already visited 
     */
    final private Set<String> myLinksAlreadyVisited;
    
    /** 
     * Data gatherer for results 
     */
    final private DataGatherer myDataGatherer;
    
    /** 
     * A list containing contents of web pages 
     */
    final private List<String> myWebpageContent;
    
    /** 
     * A map of keywords and values found 
     */
    private Map<String, Integer> theWordMap;
    
    /** 
     * total word count 
     */
    private int theWordCount;
    
    /** 
     * A variable to keep track of URL count. 
     */
    private int theURLCount;
	
    /**
     * Initialize the page parser. 
     * 
     * @param theRetrieveQueue New Hyperlinks Queue 
     * @param theLinksAlreadyVisited URL's that have already been visited. Thread safe.
     * @param words	KeyWords to search for. Thread safe.
     * @param theDataGatherer The data gatherer the receives the resulting statistics.
     * @param theWebpageContent A list of strings that contain data from an HTML or text page. 
     */
	public PageParser(BlockingQueue<String> theRetrieveQueue, Set<String> theLinksAlreadyVisited,
					  Set<String> words, DataGatherer theDataGatherer, List<String> theWebpageContent) {
	   myRetrieveQueue = theRetrieveQueue;
	   myLinksAlreadyVisited =	theLinksAlreadyVisited;
	   myDataGatherer =	theDataGatherer;
	   myWebpageContent = theWebpageContent;
	   theWordMap =  new HashMap<String, Integer>();
	   
	   theWordCount = 0;
	   theURLCount = 0;
	   
	   synchronized(words) {
		   for(String word : words){
			   theWordMap.put(word.toLowerCase(), 0);
		   }
	   }
	}

	/**
	 * Method to parse content
	 * @param theLine The Line to parse. 
	 */
	private void parseContent(final String theLine) {
		String[] tok = theLine.split(DELIMITER);
		
		for(int i = 0; i < tok.length; i++){
			String word = tok[i].toLowerCase().trim();
			if(!word.isEmpty()){
				if(theWordMap.containsKey(word)){
					theWordMap.put(word, theWordMap.get(word) + 1);
				}
				theWordCount++;
			}
		}
	}
	
	/**
	 * Method to Parse Links
	 * @param thePage Page to parse 
	 * @param theTag Anchor Tag
	 */
	private void parseLink(final String thePage, final String theTag) {
		
		//Find out what kind of quote was used.
		char aDblQuote = '"';
		if(!theTag.contains(aDblQuote + "")){
			aDblQuote = '\'';
		}
		//verify anchor tag
		if(theTag.length() > 3 && theTag.charAt(1) == 'a' && theTag.charAt(2) == ' ') {
			int index = theTag.indexOf("href");
			//If an href is found.
			if(index != -1) {
				theURLCount++;

				int index2 = theTag.indexOf(aDblQuote, index + 6);
				String href = !(index2 == -1) ? theTag.substring(index + 6, index2).trim() : "";
				String url = "";
				//Only follow designated links
				if(!href.endsWith(".txt") && !href.endsWith(".html") &&
				   !href.endsWith("/") && !href.endsWith(".htm") ) {
					return;
				}
				
				if(href.startsWith("http://") || href.startsWith("https://")) {
					url = href;
				}else if(href.isEmpty() || href.charAt(0) == '#') {
					return;
				}else {
					URL absolute = null;
					try {
						URL base = new URL(thePage);
						absolute = new URL(base, href);
					} catch (MalformedURLException e) {
						e.printStackTrace();
					}
					url = absolute.toExternalForm();
				}

				//If we have not visited the url, then add it.
				if(url.startsWith("http") && myLinksAlreadyVisited.add(url)) {
					myRetrieveQueue.add(url);
				}		
			}
		}
	}
	
	/**
	 * Run Method to parse the web pages. Send statistics to DataGather
	 * and any links found sent to the Links Queue.
	 */
	@Override
	public void run() {
		String theWebPage = "";
		long theStartTime = System.currentTimeMillis();
		String tag = "";

		boolean aBodyTagFound = false;	
		boolean aFinishTagFound = false;
		boolean isText = false;
		
		for (String line : myWebpageContent) {
			if(theWebPage.isEmpty()) {
				theWebPage = line;
				if(line.endsWith(".txt")){
					isText = true;
				}
				continue;
			}
			boolean breakOut = false;	
			line = line.trim();			
			while (!line.isEmpty() && !breakOut) {	
				
				line = line.trim();

				if((line.charAt(0) == '<' && !isText)|| aFinishTagFound) {
					boolean isClosing = line.length() > 2 && !aFinishTagFound ? line.charAt(1) == '/' : false;
					
					if(!isClosing) {
						int index = 0;
						
						while(line.length() > index && 
							  line.charAt(index) != ' ' && 
							  line.charAt(index) != '>'){
							index++;
						}
						
						//Grab the actual tag.
						tag = aFinishTagFound ? tag : line.substring(1, index);
						
						//Skip a line if there is no end.
						if(line.indexOf('>') == -1){
							//Skip the line if it does not close on this line.
							breakOut = true;
							aFinishTagFound = true;
							continue;
						}
						
						//Make sure we don't have a self closing tag, and
						//that it's not a comment.
						if(line.indexOf('>') == 0 || (line.charAt(line.indexOf('>') - 1) != '/' &&
						   !tag.startsWith("!--"))){
							if(tag.equalsIgnoreCase("a")) {
								parseLink(theWebPage, line.substring(0, line.indexOf('>') + 1));
							}else if(tag.equalsIgnoreCase("body")) {
								aBodyTagFound = true;
							}
						}			
					}
					line = line.substring(line.indexOf('>') + 1);
					aFinishTagFound = false;
				}else{			
					//We have content.
					int index = line.indexOf('<');
					if(index == -1){
						if(aBodyTagFound && !tag.equalsIgnoreCase("script")) {
							//only content
							parseContent(line);			
						}
						line = "";
					} else {
						if(aBodyTagFound && !tag.equalsIgnoreCase("script")) {
							parseContent(line.substring(0, index));
						}
						line = line.substring(index);
					}
				}
			}
		}

		//Add all of our data to the data gatherer.
		myDataGatherer.addData(theWordMap, theWordCount, theURLCount, theWebPage, 
				             (int)(System.currentTimeMillis() - theStartTime));
		
		setChanged();
		notifyObservers();
	}
}
